from collections import defaultdict, Counter
import gzip, json, os, csv
from math import sqrt, log, log10, pow

_DESCRIPTIONS_PATH_ = "./DATASET/DESCRIPTIONS/"
_FREQUENCIES_PATH_ = "./DATASET/FREQUENCIES/"
_VOCABULARY_PATH_ = "./DATASET/VOCABULARY"

def calculate_frequencies(text):
	return Counter()

def read_descriptions_into_dataset():
	orfile = gzip.open("descriptions.txt.gz",'r')
	buf = orfile.readline()
	while buf != "":
		prodID = buf.strip().split(":")[1]
		buf = orfile.readline()
		description = ":".join(buf.strip().split(":")[1:])
		buf = orfile.readline()
		target = open(_DESCRIPTIONS_PATH_+prodID,'w')
		target.write(description)
		target.close()
		buf = orfile.readline()

def calculate_frequencies(PRODUCTS):
	for product in PRODUCTS:
		source = open(_DESCRIPTIONS_PATH_+product,'r')
		description = source.readline()
		frequencies = calculate_frequencies(description)
		target = open(_FREQUENCIES_PATH_+product,'w')
		target.write(json.dumps(frequencies))
		target.close()
		source.close()

def calculate_vocabulary(PRODUCTS):
	vocabulary = Counter()
	for product in PRODUCTS:
		source = open(_FREQUENCIES_PATH_+product,'r')
		frequencies = Counter(json.loads(source.readline()))
		for word in frequencies.keys():
			vocabulary[word] += 1
	target = open(_VOCABULARY_PATH_,'w')
	target.write(json.dumps(vocabulary))
	target.close()


